{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:21.149966Z",
     "start_time": "2019-04-02T15:14:18.754055Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "train_raw = mz.datasets.toy.load_data('train')\n",
    "dev_raw = mz.datasets.toy.load_data('dev')\n",
    "test_raw = mz.datasets.toy.load_data('test')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-14T03:48:22.915655Z",
     "start_time": "2019-01-14T03:48:22.879130Z"
    }
   },
   "source": [
    "## basic Usage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A couple things are needed by the tuner:\n",
    " - a model with a parameters filled\n",
    " - preprocessed training data\n",
    " - preprocessed testing data\n",
    " \n",
    "Since MatchZoo models have pre-defined hyper-spaces, the tuner can start tuning right away once you have the data ready."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### prepare the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:21.269885Z",
     "start_time": "2019-04-02T15:14:21.152321Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preprocessor = mz.models.DenseBaseline.get_default_preprocessor()\n",
    "train = preprocessor.fit_transform(train_raw, verbose=0)\n",
    "dev = preprocessor.transform(dev_raw, verbose=0)\n",
    "test = preprocessor.transform(test_raw, verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-14T10:33:22.859605Z",
     "start_time": "2019-01-14T10:33:22.857225Z"
    }
   },
   "source": [
    "### prepare the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:21.274582Z",
     "start_time": "2019-04-02T15:14:21.271546Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = mz.models.DenseBaseline()\n",
    "model.params['input_shapes'] = preprocessor.context['input_shapes']\n",
    "model.params['task'] = mz.tasks.Ranking()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-14T10:33:44.646987Z",
     "start_time": "2019-01-14T10:33:44.644671Z"
    }
   },
   "source": [
    "### start tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.740906Z",
     "start_time": "2019-04-02T15:14:21.278631Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run #1\n",
      "Score: 0.08333333333333333\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 404\n",
      "mlp_num_layers                4\n",
      "mlp_num_fan_out               88\n",
      "mlp_activation_func           relu\n",
      "\n",
      "Run #2\n",
      "Score: 0.125\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 17\n",
      "mlp_num_layers                3\n",
      "mlp_num_fan_out               96\n",
      "mlp_activation_func           relu\n",
      "\n",
      "Run #3\n",
      "Score: 0.125\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 293\n",
      "mlp_num_layers                2\n",
      "mlp_num_fan_out               120\n",
      "mlp_activation_func           relu\n",
      "\n",
      "Run #4\n",
      "Score: 0.5\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 458\n",
      "mlp_num_layers                2\n",
      "mlp_num_fan_out               56\n",
      "mlp_activation_func           relu\n",
      "\n",
      "Run #5\n",
      "Score: 0.16666666666666666\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 313\n",
      "mlp_num_layers                2\n",
      "mlp_num_fan_out               36\n",
      "mlp_activation_func           relu\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tuner = mz.auto.Tuner(\n",
    "    params=model.params,\n",
    "    train_data=train,\n",
    "    test_data=dev,\n",
    "    num_runs=5\n",
    ")\n",
    "results = tuner.tune()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### view the best hyper-parameter set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.756749Z",
     "start_time": "2019-04-02T15:14:25.743758Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'#': 4,\n",
       " 'params': <matchzoo.engine.param_table.ParamTable at 0x13dfcfef0>,\n",
       " 'sample': {'mlp_num_fan_out': 56.0,\n",
       "  'mlp_num_layers': 2.0,\n",
       "  'mlp_num_units': 458.0},\n",
       " 'score': 0.5}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results['best']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.780015Z",
     "start_time": "2019-04-02T15:14:25.759362Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Description</th>\n",
       "      <th>Value</th>\n",
       "      <th>Hyper-Space</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>model_class</td>\n",
       "      <td>Model class. Used internally for save/load. Ch...</td>\n",
       "      <td>&lt;class 'matchzoo.models.dense_baseline.DenseBa...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>input_shapes</td>\n",
       "      <td>Dependent on the model and data. Should be set...</td>\n",
       "      <td>[(30,), (30,)]</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>task</td>\n",
       "      <td>Decides model output shape, loss, and metrics.</td>\n",
       "      <td>Ranking Task</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>optimizer</td>\n",
       "      <td>None</td>\n",
       "      <td>adam</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>with_multi_layer_perceptron</td>\n",
       "      <td>A flag of whether a multiple layer perceptron ...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>mlp_num_units</td>\n",
       "      <td>Number of units in first `mlp_num_layers` layers.</td>\n",
       "      <td>458</td>\n",
       "      <td>quantitative uniform distribution in  [16, 512...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>mlp_num_layers</td>\n",
       "      <td>Number of layers of the multiple layer percetron.</td>\n",
       "      <td>2</td>\n",
       "      <td>quantitative uniform distribution in  [1, 5), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>mlp_num_fan_out</td>\n",
       "      <td>Number of units of the layer that connects the...</td>\n",
       "      <td>56</td>\n",
       "      <td>quantitative uniform distribution in  [4, 128)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>mlp_activation_func</td>\n",
       "      <td>Activation function used in the multiple layer...</td>\n",
       "      <td>relu</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Name  \\\n",
       "0                  model_class   \n",
       "1                 input_shapes   \n",
       "2                         task   \n",
       "3                    optimizer   \n",
       "4  with_multi_layer_perceptron   \n",
       "5                mlp_num_units   \n",
       "6               mlp_num_layers   \n",
       "7              mlp_num_fan_out   \n",
       "8          mlp_activation_func   \n",
       "\n",
       "                                         Description  \\\n",
       "0  Model class. Used internally for save/load. Ch...   \n",
       "1  Dependent on the model and data. Should be set...   \n",
       "2     Decides model output shape, loss, and metrics.   \n",
       "3                                               None   \n",
       "4  A flag of whether a multiple layer perceptron ...   \n",
       "5  Number of units in first `mlp_num_layers` layers.   \n",
       "6  Number of layers of the multiple layer percetron.   \n",
       "7  Number of units of the layer that connects the...   \n",
       "8  Activation function used in the multiple layer...   \n",
       "\n",
       "                                               Value  \\\n",
       "0  <class 'matchzoo.models.dense_baseline.DenseBa...   \n",
       "1                                     [(30,), (30,)]   \n",
       "2                                       Ranking Task   \n",
       "3                                               adam   \n",
       "4                                               True   \n",
       "5                                                458   \n",
       "6                                                  2   \n",
       "7                                                 56   \n",
       "8                                               relu   \n",
       "\n",
       "                                         Hyper-Space  \n",
       "0                                               None  \n",
       "1                                               None  \n",
       "2                                               None  \n",
       "3                                               None  \n",
       "4                                               None  \n",
       "5  quantitative uniform distribution in  [16, 512...  \n",
       "6  quantitative uniform distribution in  [1, 5), ...  \n",
       "7  quantitative uniform distribution in  [4, 128)...  \n",
       "8                                               None  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results['best']['params'].to_frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## understading hyper-space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`model.params.hyper_space` reprensents the model's hyper-parameters search space, which is the cross-product of individual hyper parameter's hyper space. When a `Tuner` builds a model, for each hyper parameter in `model.params`, if the hyper-parameter has a hyper-space, then a sample will be taken in the space. However, if the hyper-parameter does not have a hyper-space, then the default value of the hyper-parameter will be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.789505Z",
     "start_time": "2019-04-02T15:14:25.783459Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mlp_num_units': <hyperopt.pyll.base.Apply at 0x13c5967f0>,\n",
       " 'mlp_num_layers': <hyperopt.pyll.base.Apply at 0x13f7982b0>,\n",
       " 'mlp_num_fan_out': <hyperopt.pyll.base.Apply at 0x13c5a2b70>}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.params.hyper_space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In a `DenseBaseline` model, only `mlp_num_units`, `mlp_num_layers`, and `mlp_num_fan_out` have pre-defined hyper-space. In other words, only these hyper-parameters will change values during a tuning. Other hyper-parameters, like `mlp_activation_func`, are fixed and will not change."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.808039Z",
     "start_time": "2019-04-02T15:14:25.791727Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "if sampled: {'mlp_num_fan_out': 60.0, 'mlp_num_layers': 4.0, 'mlp_num_units': 146.0} \n",
      "\n",
      "the built model will have:\n",
      "\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 146\n",
      "mlp_num_layers                4\n",
      "mlp_num_fan_out               60\n",
      "mlp_activation_func           relu \n",
      "\n",
      "\n",
      "\n",
      "if sampled: {'mlp_num_fan_out': 64.0, 'mlp_num_layers': 1.0, 'mlp_num_units': 436.0} \n",
      "\n",
      "the built model will have:\n",
      "\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 436\n",
      "mlp_num_layers                1\n",
      "mlp_num_fan_out               64\n",
      "mlp_activation_func           relu \n",
      "\n",
      "\n",
      "\n",
      "if sampled: {'mlp_num_fan_out': 44.0, 'mlp_num_layers': 4.0, 'mlp_num_units': 48.0} \n",
      "\n",
      "the built model will have:\n",
      "\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 48\n",
      "mlp_num_layers                4\n",
      "mlp_num_fan_out               44\n",
      "mlp_activation_func           relu \n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def sample_and_build(params):\n",
    "    sample = mz.hyper_spaces.sample(params.hyper_space)\n",
    "    print('if sampled:', sample, '\\n')\n",
    "    params.update(sample)\n",
    "    print('the built model will have:\\n')\n",
    "    print(params, '\\n\\n\\n')\n",
    "\n",
    "for _ in range(3):\n",
    "    sample_and_build(model.params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T07:12:19.698787Z",
     "start_time": "2019-04-02T07:12:19.693403Z"
    },
    "collapsed": true
   },
   "source": [
    "This is similar to the process of a tuner sampling model hyper-parameters, but with one key difference: a tuner's hyper-space is **suggestive**. This means the sampling process in a tuner is not truely random but skewed. Scores of the past samples affect future choices: a tuner with more runs knows better about its hyper-space, and take samples in a way that will likely yields better scores.\n",
    "\n",
    "For more details, consult tuner's backend: [hyperopt](http://hyperopt.github.io/hyperopt/), and the search algorithm tuner uses: [Tree of Parzen Estimators (TPE)](https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hyper-spaces can also be represented in a human-readable format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.813201Z",
     "start_time": "2019-04-02T15:14:25.809832Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "quantitative uniform distribution in  [16, 512), with a step size of 1\n"
     ]
    }
   ],
   "source": [
    "print(model.params.get('mlp_num_units').hyper_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.830261Z",
     "start_time": "2019-04-02T15:14:25.816717Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Hyper-Space</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>model_class</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>input_shapes</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>task</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>optimizer</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>with_multi_layer_perceptron</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>mlp_num_units</td>\n",
       "      <td>quantitative uniform distribution in  [16, 512...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>mlp_num_layers</td>\n",
       "      <td>quantitative uniform distribution in  [1, 5), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>mlp_num_fan_out</td>\n",
       "      <td>quantitative uniform distribution in  [4, 128)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>mlp_activation_func</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Name  \\\n",
       "0                  model_class   \n",
       "1                 input_shapes   \n",
       "2                         task   \n",
       "3                    optimizer   \n",
       "4  with_multi_layer_perceptron   \n",
       "5                mlp_num_units   \n",
       "6               mlp_num_layers   \n",
       "7              mlp_num_fan_out   \n",
       "8          mlp_activation_func   \n",
       "\n",
       "                                         Hyper-Space  \n",
       "0                                               None  \n",
       "1                                               None  \n",
       "2                                               None  \n",
       "3                                               None  \n",
       "4                                               None  \n",
       "5  quantitative uniform distribution in  [16, 512...  \n",
       "6  quantitative uniform distribution in  [1, 5), ...  \n",
       "7  quantitative uniform distribution in  [4, 128)...  \n",
       "8                                               None  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.params.to_frame()[['Name', 'Hyper-Space']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### setting hyper-space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What if I want the tuner to choose `optimizer` among `adam`, `adagrad`, and `rmsprop`?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.835257Z",
     "start_time": "2019-04-02T15:14:25.832370Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.params.get('optimizer').hyper_space = mz.hyper_spaces.choice(['adam', 'adagrad', 'rmsprop'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.879425Z",
     "start_time": "2019-04-02T15:14:25.837305Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mlp_num_fan_out': 24.0, 'mlp_num_layers': 4.0, 'mlp_num_units': 439.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 112.0, 'mlp_num_layers': 3.0, 'mlp_num_units': 299.0, 'optimizer': 'rmsprop'}\n",
      "{'mlp_num_fan_out': 76.0, 'mlp_num_layers': 3.0, 'mlp_num_units': 353.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 120.0, 'mlp_num_layers': 3.0, 'mlp_num_units': 496.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 64.0, 'mlp_num_layers': 2.0, 'mlp_num_units': 290.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 72.0, 'mlp_num_layers': 5.0, 'mlp_num_units': 360.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 68.0, 'mlp_num_layers': 1.0, 'mlp_num_units': 100.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 20.0, 'mlp_num_layers': 2.0, 'mlp_num_units': 511.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 92.0, 'mlp_num_layers': 2.0, 'mlp_num_units': 18.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 128.0, 'mlp_num_layers': 5.0, 'mlp_num_units': 297.0, 'optimizer': 'adagrad'}\n"
     ]
    }
   ],
   "source": [
    "for _ in range(10):\n",
    "    print(mz.hyper_spaces.sample(model.params.hyper_space))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What about setting `mlp_num_layers` to a fixed value of 2?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.886117Z",
     "start_time": "2019-04-02T15:14:25.881809Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.params['mlp_num_layers'] = 2\n",
    "model.params.get('mlp_num_layers').hyper_space = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:25.926672Z",
     "start_time": "2019-04-02T15:14:25.888999Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mlp_num_fan_out': 100.0, 'mlp_num_units': 168.0, 'optimizer': 'adam'}\n",
      "{'mlp_num_fan_out': 68.0, 'mlp_num_units': 487.0, 'optimizer': 'adam'}\n",
      "{'mlp_num_fan_out': 124.0, 'mlp_num_units': 356.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 116.0, 'mlp_num_units': 499.0, 'optimizer': 'rmsprop'}\n",
      "{'mlp_num_fan_out': 32.0, 'mlp_num_units': 120.0, 'optimizer': 'rmsprop'}\n",
      "{'mlp_num_fan_out': 84.0, 'mlp_num_units': 442.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 92.0, 'mlp_num_units': 142.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 68.0, 'mlp_num_units': 472.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 32.0, 'mlp_num_units': 343.0, 'optimizer': 'adagrad'}\n",
      "{'mlp_num_fan_out': 68.0, 'mlp_num_units': 19.0, 'optimizer': 'adam'}\n"
     ]
    }
   ],
   "source": [
    "for _ in range(10):\n",
    "    print(mz.hyper_spaces.sample(model.params.hyper_space))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### using callbacks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To save the model during the tuning process, use `mz.auto.tuner.callbacks.SaveModel`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:27.885783Z",
     "start_time": "2019-04-02T15:14:25.929438Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: `tune` does not affect the tuner's inner state, so\n",
      "                each new call to `tune` starts fresh. In other words,\n",
      "                hyperspaces are suggestive only within the same `tune` call.\n",
      "Run #1\n",
      "Score: 0.07142857142857142\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     rmsprop\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 360\n",
      "mlp_num_layers                2\n",
      "mlp_num_fan_out               92\n",
      "mlp_activation_func           relu\n",
      "\n",
      "Run #2\n",
      "Score: 0.08333333333333333\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adagrad\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 99\n",
      "mlp_num_layers                2\n",
      "mlp_num_fan_out               84\n",
      "mlp_activation_func           relu\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tuner.num_runs = 2\n",
    "tuner.callbacks.append(mz.auto.tuner.callbacks.SaveModel())\n",
    "results = tuner.tune()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This will save all built models to your `mz.USER_TUNED_MODELS_DIR`, and can be loaded by:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:28.378822Z",
     "start_time": "2019-04-02T15:14:27.887573Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matchzoo.models.dense_baseline.DenseBaseline at 0x13deda6a0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model_id = results['best']['model_id']\n",
    "mz.load_model(mz.USER_TUNED_MODELS_DIR.joinpath(best_model_id))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To load a pre-trained embedding layer into a built model during a tuning process, use `mz.auto.tuner.callbacks.LoadEmbeddingMatrix`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:28.468787Z",
     "start_time": "2019-04-02T15:14:28.380895Z"
    }
   },
   "outputs": [],
   "source": [
    "toy_embedding = mz.datasets.toy.load_embedding()\n",
    "preprocessor = mz.models.DUET.get_default_preprocessor()\n",
    "train = preprocessor.fit_transform(train_raw, verbose=0)\n",
    "dev = preprocessor.transform(dev_raw, verbose=0)\n",
    "params = mz.models.DUET.get_default_params()\n",
    "params['task'] = mz.tasks.Ranking()\n",
    "params.update(preprocessor.context)\n",
    "params['embedding_output_dim'] = toy_embedding.output_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:28.508952Z",
     "start_time": "2019-04-02T15:14:28.474349Z"
    }
   },
   "outputs": [],
   "source": [
    "embedding_matrix = toy_embedding.build_matrix(preprocessor.context['vocab_unit'].state['term_index'])\n",
    "load_embedding_matrix_callback = mz.auto.tuner.callbacks.LoadEmbeddingMatrix(embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:31.489312Z",
     "start_time": "2019-04-02T15:14:28.517138Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run #1\n",
      "Score: 0.125\n",
      "model_class                   <class 'matchzoo.models.duet.DUET'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_embedding                True\n",
      "embedding_input_dim           285\n",
      "embedding_output_dim          2\n",
      "embedding_trainable           True\n",
      "lm_filters                    32\n",
      "lm_hidden_sizes               [32]\n",
      "dm_filters                    32\n",
      "dm_kernel_size                3\n",
      "dm_q_hidden_size              32\n",
      "dm_d_mpool                    3\n",
      "dm_hidden_sizes               [32]\n",
      "padding                       same\n",
      "activation_func               relu\n",
      "dropout_rate                  0.14\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tuner = mz.auto.tuner.Tuner(\n",
    "    params=params,\n",
    "    train_data=train,\n",
    "    test_data=dev,\n",
    "    num_runs=1\n",
    ")\n",
    "tuner.callbacks.append(load_embedding_matrix_callback)\n",
    "results = tuner.tune()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### make your own callbacks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To build your own callbacks, inherit `mz.auto.tuner.callbacks.Callback` and overrides corresponding methods.\n",
    "\n",
    "A run proceeds in the following way:\n",
    "\n",
    "- run start (callback)\n",
    "- build model\n",
    "- build end (callback)\n",
    "- fit and evaluate model\n",
    "- collect result\n",
    "- run end (callback)\n",
    "\n",
    "This process is repeated for `num_runs` times in a tuner."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For example, say I want to verify if my embedding matrix is correctly loaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:31.494610Z",
     "start_time": "2019-04-02T15:14:31.490880Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class ValidateEmbedding(mz.auto.tuner.callbacks.Callback):\n",
    "    def __init__(self, embedding_matrix):\n",
    "        self._matrix = embedding_matrix\n",
    "        \n",
    "    def on_build_end(self, tuner, model):\n",
    "        loaded_matrix = model.get_embedding_layer().get_weights()[0]\n",
    "        if np.isclose(self._matrix, loaded_matrix).all():\n",
    "            print(\"Yes! The my embedding is correctly loaded!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:31.502034Z",
     "start_time": "2019-04-02T15:14:31.496765Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "validate_embedding_matrix_callback = ValidateEmbedding(embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-02T15:14:35.199659Z",
     "start_time": "2019-04-02T15:14:31.503862Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Yes! The my embedding is correctly loaded!\n",
      "Run #1\n",
      "Score: 0.08333333333333333\n",
      "model_class                   <class 'matchzoo.models.duet.DUET'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_embedding                True\n",
      "embedding_input_dim           285\n",
      "embedding_output_dim          2\n",
      "embedding_trainable           True\n",
      "lm_filters                    32\n",
      "lm_hidden_sizes               [32]\n",
      "dm_filters                    32\n",
      "dm_kernel_size                3\n",
      "dm_q_hidden_size              32\n",
      "dm_d_mpool                    3\n",
      "dm_hidden_sizes               [32]\n",
      "padding                       same\n",
      "activation_func               relu\n",
      "dropout_rate                  0.44\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tuner = mz.auto.tuner.Tuner(\n",
    "    params=params,\n",
    "    train_data=train,\n",
    "    test_data=dev,\n",
    "    num_runs=1,\n",
    "    callbacks=[load_embedding_matrix_callback, validate_embedding_matrix_callback]\n",
    ")\n",
    "tuner.callbacks.append(load_embedding_matrix_callback)\n",
    "results = tuner.tune()"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "matchzoo",
   "language": "python",
   "name": "matchzoo"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {
    "height": "1679px",
    "left": "0px",
    "right": "909px",
    "top": "161px",
    "width": "171px"
   },
   "toc_section_display": "none",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
